# -*- coding: utf-8 -*-
# @Time    : 2019/2/26 11:16
# @Author  : yueconger
# @File    : faxin_wx.py
import re
import json
import requests
import time
from pymongo import MongoClient
import conf
import et_proxy


class FaxinWechatApplet(object):
    def __init__(self):
        self.answer_url = 'https://ask.gridsumdissector.com/master/api/MiniIQA/Answer'
        self.view_point_url = 'https://ask.gridsumdissector.com/master/api/MiniIQA/ViewPointInfo?Id={ViewPointId}'
        self.law_url = 'https://ask.gridsumdissector.com/master/api/MiniIQA/LawInfo?Id={LawId}'
        self.headers = {
            'charset': 'utf-8',
            'Accept-Encoding': 'gzip',
            'referer': 'https://servicewechat.com/wx4e20ce6ae4b67e59/6/page-frame.html',
            'content-type': 'application/json',
            'User-Agent': 'Mozilla/5.0 (Linux; Android 9; MI MAX 3 Build/PKQ1.181007.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 MicroMessenger/7.0.3.1400(0x2700033A) Process/appbrand2 NetType/WIFI Language/zh_CN',
            'Host': 'ask.gridsumdissector.com',
            'Connection': 'Keep-Alive',
        }

    def start_request(self, question_set):
        start_url = 'https://ask.gridsumdissector.com/master/api/MiniIQA/GetHotQuestions'
        body = self.get_body(start_url)
        print(body)
        try:
            questions = body['Data']['Questions']
        except Exception as e:
            print('起始页访问失败', e)
        else:
            print(questions)
            if questions is not None:
                print('------')
                for question in questions:
                    question_set.add(question)
                    self.backup_txt(question)
                    data = {"Question": question}
                    res = self.post_body(self.answer_url, data)
                    print(res)
                    self.parse(res, question, question_set)

    def parse(self, res, post_qestion, question_set):
        print('=============')
        success = res['Success']
        if success:
            if 'Data' in res:
                Data = res['Data']
                AnswerType = Data['AnswerType']
                Content = Data['Content']
                if AnswerType == 207:
                    # 法律法规
                    if 'LawInfo' in Content:
                        LawId = Content['LawInfo']['Id']
                        LawName = Content['LawInfo']['Name']

                        law_url = self.law_url.format(LawId=LawId)
                        get_body = self.get_body(law_url)
                        self.DB_Mongo(get_body, '法律法规库')
                        print('法律', LawName, '已保存!')

                else:
                    # 问答 (字典格式)
                    QA = Content['QuestionAndAnswer'] if 'QuestionAndAnswer' in Content else {}
                    if len(QA) > 0:
                        qa_item = {}
                        question = QA['Question']

                        question_set.add(question)
                        answer = QA['Answer']
                        qa_item['question'] = question
                        qa_item['answer'] = answer

                        # 权威数据
                        AS = Content['AuthoritativeSource'] if 'AuthoritativeSource' in Content else {}
                        if len(AS) > 0:
                            # 权威数据_法律依据
                            LegalBasis = AS['LegalBasis']  # 列表
                            qa_item['LegalBasis'] = LegalBasis
                            # if len(LegalBasis) > 0:
                            #     for law in LegalBasis:
                            #         law_id = law['Id']
                            #         law_name = law['Name']

                            # 权威数据_参考观点
                            SourceViewPoint = AS['SourceViewPoint']
                            ViewPointId = SourceViewPoint['Id']
                            view_point_url = self.view_point_url.format(ViewPointId=ViewPointId)
                            view_res = self.get_body(view_point_url)

                            qa_item['view_point'] = view_res
                            # 权威数据_案件 无数据 不采集

                        self.DB_Mongo(qa_item, '问答库_2')
                        print(question, '已保存')

                    # 法条 (列表格式)
                    Provision = Content['ProvisionInfos'] if 'ProvisionInfos' in Content else []
                    if len(Provision) > 0:
                        ProvisionInfos = Content['ProvisionInfos']  # 法条信息 直接存储, 后期处理
                        provison_item = {'question': post_qestion, 'provision': ProvisionInfos}
                        self.DB_Mongo(provison_item, '法条库')

                    # 相似问题 (列表格式)
                    SimilarQuestions = Content['SimilarQuestions'] if 'SimilarQuestions' in Content else []
                    if len(SimilarQuestions) > 0:
                        for squestion in SimilarQuestions:
                            if squestion in question_set:
                                print('相似问题:%s 已存在, 不再采集' % squestion)
                                continue
                            else:
                                question_set.add(squestion)
                                self.backup_txt(squestion)
                                json_data = {"Question": squestion}
                                res = self.post_body(self.answer_url, json_data)
                                self.parse(res, squestion, question_set)
            else:
                print('采集数据出错:')
        else:
            print('采集出错')

    def get_body(self, url):
        response = requests.get(url, headers=self.headers)
        html = response.content.decode()
        try:
            res = json.loads(html)
        except Exception as e:
            print('代理失效,需要更换', e)
            state = et_proxy.change_proxy()
            print(state)
            res = self.get_body(url)
            time.sleep(5)
        return res

    def post_body(self, url, json_data):
        response = requests.post(url, headers=self.headers, json=json_data)
        html = response.content.decode()
        try:
            res = json.loads(html)
        except Exception as e:
            print('代理失效,需要更换', e)
            state = et_proxy.change_proxy()
            print(state)
            res = self.post_body(url, json_data)
            time.sleep(5)
        return res

    def DB_Mongo(self, item, doc_name):
        """
        :param item: 类json数据存入MongoDB中
        :param doc_name: 表名
        """
        client = MongoClient(
            host=conf.MONGDB_HOST,
            port=conf.PORT
        )
        db = client[conf.MONGODB_DBNAME]
        collection = db[doc_name]
        collection.insert(item)

    def backup_txt(self, question):
        with open('qa_pool.txt', 'a+', encoding='utf-8') as f:
            f.write(question + '\n')

        # with open('qa_pool.txt', 'r', encoding='utf-8') as f:
        #     con = f.read()
        #
        # with open('qa_pool.txt.bak', 'w', encoding='utf-8') as f:
        #     f.write(con)


if __name__ == '__main__':
    faxin = FaxinWechatApplet()
    question_set = set()
    with open('qa_pool.txt', 'r', encoding='utf-8') as f:
        con = f.readlines()
    for q in con:
        question = q.strip()
        question_set.add(question)
    faxin.start_request(question_set)
